package xxx.yyy.zzz.movie;

import org.apache.commons.lang3.StringUtils;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Pattern;

/**
 * https://www.1905.com/
 * https://www.1905.com/mdb/film/list/year-2019/
 * https://www.1905.com/mdb/film/list/year-2019/o0d0p2.html
 * https://www.1905.com/mdb/film/list/year-2019/o0d0p3.html
 * https://www.1905.com/mdb/film/list/year-2019/o0d0p4.html
 * https://www.1905.com/mdb/film/list/year-2019/o0d0p5.html
 * https://www.1905.com/mdb/film/list/year-2019/o0d0p6.html
 * https://www.1905.com/mdb/film/list/year-2019/o0d0p7.html
 * https://www.1905.com/mdb/film/list/year-2019/o0d0p8.html
 * https://www.1905.com/mdb/film/list/year-2019/o0d0p9.html
 * https://www.1905.com/mdb/film/list/year-2019/o0d0p10.html
 */
public class MovieIDProcessor implements PageProcessor {

    private Set<String> ids;

    public MovieIDProcessor() {
        this.ids = new HashSet<>();
    }

    // 抓取网站的相关配置，包括：编码、抓取间隔、重试次数等
    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);

    @Override
    public void process(Page page) {
        List<Selectable> list = page.getHtml().css("li.fl.line").nodes();
        for (Selectable selectable : list) {
            String content = selectable.xpath("//li/a/@href").get();
            String[] tokens = StringUtils.split(content, "/");
            if (tokens != null
                    && tokens.length != 0
                    && Pattern.matches("\\d+", tokens[tokens.length-1]))
                ids.add(tokens[tokens.length - 1]);
        }

        page.putField("ids", ids);
    }

    public Site getSite() {
        return site;
    }
}
